package news.crawler;

import java.util.List;

import edu.uci.ics.crawler4j.crawler.CrawlController;

public class Controller {
	
	public static void main(String[] args) throws Exception {
		String rootFolder = "/home/vamshi/Downloads/tmp-news/";
		int numberOfCrawlers = 3;
		String storageFolder = "/home/vamshi/Downloads/tmp-news/content/";

		String[] crawlDomains = new String[] { "http://www.nytimes.com/",
				"http://news.google.com/", "http://www.greatandhra.com/" };

		CrawlController controller = new CrawlController(rootFolder);
 
		for (String domain : crawlDomains) {
			controller.addSeed(domain);
		}
		
		// Be polite:
		controller.setPolitenessDelay(200);
 				
		MyImageCrawler.configure(crawlDomains, storageFolder);
		// MyCrawler.configure(crawlDomains);

		controller.start(MyImageCrawler.class, numberOfCrawlers);
		
		List<Object> crawlersLocalData = controller.getCrawlersLocalData();
		long totalLinks = 0;
		long totalTextSize = 0;
		int totalProcessedPages = 0;
		for (Object localData : crawlersLocalData) {
			CrawlStat stat = (CrawlStat) localData;
			totalLinks += stat.getTotalLinks();
			totalTextSize += stat.getTotalTextSize();
			totalProcessedPages += stat.getTotalProcessedPages();
		}
		System.out.println("Aggregated Statistics:");
		System.out.println("   Processed Pages: " + totalProcessedPages);
		System.out.println("   Total Links found: " + totalLinks);
		System.out.println("   Total Text Size: " + totalTextSize);
	}

}
 